#!/usr/bin/env python 
# -*- coding: utf-8 -*- 
'''
Created on 2016年5月23日
@author: zhaoxiang
根据 论坛，问答，行程，目的地，锦囊，折扣，，点评，酒店等业务表
解析出用户相关行为内容(id)，
并根据uid(guid) 匹配出dora_id
统计city,dountry，poi
'''
import sys,os
sys.path.append('/home/hadoop/udf_data/python/user_track')
os.environ['PYTHON_EGG_CACHE'] = '/tmp/hadoop/.python-eggs/'
os.environ['PYTHON_EGG_DIR']='/tmp/hadoop/.python-eggs/'
import datetime,format_time
import json
import jieba 
import config
from config import  DICT_FILE
import ProductTag
from DB import *
import gua
from util_func import dateToTimestamp
from MongoClient import MongoClient

reload(sys)
sys.setdefaultencoding("utf8")

doraid_views={}

#==============================
#折扣
#===============================
def buy_lm(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer["host"], gua.conn_qyer["username"], gua.conn_qyer["password"], gua.conn_qyer["db"], gua.conn_qyer["port"])
    qyer_db.execute("select uid, lid from qyer_orderform.orderform where payment = 1 and datetime >= %s and datetime < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result["uid"]
        lid = result["lid"]
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if uid_result : 
            dora_id = uid_result["dora_id"]
            key=(dora_id,"buy_lm")
            if key not in doraid_views : 
                doraid_views[key] = {"uids":set(), "buy_lm":[]}
            if uid:
                doraid_views[key]["uids"].add(uid)
            doraid_views[key]["buy_lm"].append(lid)
    return doraid_views

#==============================
#酒店预订
#===============================
black_hotel_users = set([191066, 4954313, 2311611, 5397676, 1569414, 4935486])         
def book_hotel(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer["host"], gua.conn_qyer["username"], gua.conn_qyer["password"], gua.conn_qyer["db"], gua.conn_qyer["port"])
    qyer_db.execute("select id, booker_hotel_id, uid, guid from qyer.business_partner_hotel_booking_order where book_time >= %s and book_time < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    for result in result_set :
        booking_hotel_id = result["booker_hotel_id"]
        qyer_db.execute("select id from qyer.business_hotel where booking_hotel_id = %s", (booking_hotel_id,))
        hotel_result = qyer_db.fetchone()
        if not hotel_result : continue
        hotel_id = hotel_result["id"]

        uid = result["uid"]
        if uid in black_hotel_users : continue
        guid = result["guid"]
        dora_id = 0
        if uid :
            uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
            if uid_result : dora_id = uid_result["dora_id"]
        if dora_id == 0 and len(guid) == 36 : 
            guid_result = mongo_db.execute("guid_doraid_1", {"_id":guid}, "selectone")
            if guid_result : dora_id = guid_result["dora_id"]

        if dora_id :
            key=(dora_id,"book_hotel")
            if key not in doraid_views : 
                doraid_views[key] = {"uids":set(), "book_hotel":[]}
            if uid : 
                doraid_views[key]["uids"].add(uid)
            doraid_views[key]["book_hotel"].append(hotel_id)
        
    qyer_db.close()
    return doraid_views

#==============================
#锦囊下载
#===============================
def down_guide(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer['host'], gua.conn_qyer['username'], gua.conn_qyer['password'], gua.conn_qyer['db'], gua.conn_qyer["port"])
    qyer_db.execute("select uid, jid from qyer.jn_down where downtime > %s and downtime <= %s and uid > 0", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result['uid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if uid_result :
            dora_id = uid_result["dora_id"]
            key=(dora_id,"d_guide")
            if key not in doraid_views :
                doraid_views[key] = {"uids":set(), "d_guide":set()}
            if uid : 
                doraid_views[key]["uids"].add(uid)
            doraid_views[key]["d_guide"].add(result["jid"])
    return doraid_views

#==============================
#搜索
#===============================
def ask_search(starttime, endtime, doraid_views) :
    jieba.load_userdict(DICT_FILE) 
    qyer_db = DB(gua.conn_qyer['host'], gua.conn_qyer['username'], gua.conn_qyer['password'], gua.conn_qyer['db'], gua.conn_qyer["port"])
    qyer_db.execute("select uid, search from qyer.ask_search_log where addtime >= %s and addtime < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result['uid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        segs = jieba.cut(result['search'])
        words = [w.encode('utf8', 'ignore') for w in segs if len(w) > 1]
        if len(words) :
            key=(dora_id,"search")
            if key not in doraid_views : 
                doraid_views[key] = {"uids":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}}
            if uid > 0 : 
                doraid_views[key]["uids"].add(uid)
            doraid_views[key]["search"]["wd"] += words
    strDate = format_time.timeStamp2DateStr(starttime)
    track_search_file =  config.hive_track_search % strDate
    if os.path.exists(track_search_file):
        with open(track_search_file,"r") as f:
            for line in f:
                rowSet = line.strip().split('\t')
                if (len(rowSet) < 5):
                    continue  
#                 stat_date = rowSet[0]  
                dora_id = long(rowSet[1])
                uidList = eval(rowSet[2])
                #解析成list：[str,str]
                parsed_content = eval(rowSet[4]) 
                key=(dora_id,"search")
                if key not in doraid_views : 
#                     print(dora_id,uidList)
                    doraid_views[key] = {"uids":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}}
                doraid_views[key]["uids"].update(map(int,uidList))
                for item in parsed_content:
                    #每个str解析成字典
                    item = eval(item)
                    for k,v in item.iteritems():
                        doraid_views[key]["search"][k] += v
    return doraid_views

#==============================
#提问
#===============================
def ask_question(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer['host'], gua.conn_qyer['username'], gua.conn_qyer['password'], gua.conn_qyer['db'], gua.conn_qyer["port"])
    qyer_db.execute("select qid, uid from qyer.ask_question where addtime >= %s and addtime < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result['uid']
        qid = result['qid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : 
            continue
        dora_id = uid_result["dora_id"]
        key=(dora_id,"ask")
        if key not in doraid_views : 
            doraid_views[key] = {"uids":set([]), "ask":[]}
        if uid > 0 : 
            doraid_views[key]["uids"].add(uid)
        doraid_views[key]["ask"].append(qid)
    return doraid_views

#==============================
#问答
#===============================
def answer_question(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer["host"], gua.conn_qyer["username"], gua.conn_qyer["password"], gua.conn_qyer["db"], gua.conn_qyer["port"])
    qyer_db.execute("select qid, uid from qyer.ask_answer where addtime >= %s and addtime < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    for result in result_set :
        uid = result["uid"]
        qid = result["qid"]
        qyer_db.execute("select uid from qyer.ask_question where qid = %s", (qid,))
        q_result = qyer_db.fetchone()
        if not q_result : 
            continue
        if q_result["uid"] == uid : 
            continue
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : 
            continue
        dora_id = uid_result["dora_id"]
        key=(dora_id,"answer")
        if key not in doraid_views : 
            doraid_views[key] = {"uids":set([]), "answer":set()}
        if uid : 
            doraid_views[key]["uids"].add(uid)
        doraid_views[key]["answer"].add(qid)
    qyer_db.close()
    return doraid_views

#==============================
#点评
#===============================
def place_comment(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer['host'], gua.conn_qyer['username'], gua.conn_qyer['password'], gua.conn_qyer['db'], gua.conn_qyer["port"])
    qyer_db.execute("select uid, cityid from place_comment where datetime > %s and datetime <= %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    for result in result_set :
        uid = result['uid']
        cityid = result['cityid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : 
            continue
        dora_id = uid_result["dora_id"]
        key=(dora_id,"p_comment")
        if key not in doraid_views : 
            doraid_views[key] = {"uids":set(), "p_comment":{"country":[], "city":[], "poi":[]}}
        if uid > 0 : 
            doraid_views[key]["uids"].add(uid)
        doraid_views[key]["p_comment"]["city"].append(cityid)

    qyer_db.execute("select uid, countryid from place_comment_country where datetime > %s and datetime <= %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    for result in result_set :
        uid = result['uid']
        countryid = result['countryid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        key=(dora_id,"p_comment")
        if key not in doraid_views : 
            doraid_views[key] = {"uids":set(), "p_comment":{"country":[], "city":[], "poi":[]}}
        if uid > 0 :
            doraid_views[key]["uids"].add(uid)
        doraid_views[key]["p_comment"]["country"].append(countryid)
            

    qyer_db.execute("select uid, poiid from poi_comment where datetime > %s and datetime <= %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    for result in result_set :
        uid = result['uid']
        poiid = result['poiid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        key=(dora_id,"p_comment")
        if key not in doraid_views : 
            doraid_views[key] = {"uids":set(), "p_comment":{"country":[], "city":[], "poi":[]}}        
        if uid > 0 :
            doraid_views[key]["uids"].add(uid)
        doraid_views[key]["p_comment"]["poi"].append(poiid)

    qyer_db.close()
    return doraid_views

#==============================
#论坛
#===============================
def coplay_thread(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer['host'], gua.conn_qyer['username'], gua.conn_qyer['password'], gua.conn_qyer['db'], gua.conn_qyer["port"])
    qyer_db.execute("select uid as authorid, id as tid from bbs.threads where type = 3 and created >= %s and created < %s", (starttime, endtime))
    #qyer_db.execute("select authorid, tid from newbbs_utf8.cdb_threads where typeid = 8 and dateline >= %s and dateline < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result['authorid']
        tid = result['tid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        key=(dora_id,"coplay")
        if key not in doraid_views :         
            doraid_views[key] = {"uids":set(), "coplay":[]}
        if uid : 
            doraid_views[key]["uids"].add(uid)
        doraid_views[key]["coplay"].append(tid)

    return doraid_views

#==============================
#论坛
#===============================
def travel_thread(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer["host"], gua.conn_qyer["username"], gua.conn_qyer["password"], gua.conn_qyer["db"], gua.conn_qyer["port"])
    qyer_db.execute("select uid as authorid, id as tid from bbs.threads where type in (1,2) and created >= %s and created < %s", (starttime, endtime))
    #qyer_db.execute("select authorid, tid from newbbs_utf8.cdb_threads where typeid = 5 and dateline >= %s and dateline < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result["authorid"]
        tid = result["tid"]
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        key=(dora_id,"p_travel")
        if key not in doraid_views :         
            doraid_views[key] = {"uids":set(), "p_travel":[]}
        if uid : 
            doraid_views[key]["uids"].add(uid)        
        doraid_views[key]["p_travel"].append(tid)
    return doraid_views

def tag_doraid_action(timeStamp, doraid_views) :
    pt = ProductTag.ProductTagger()
    ra_db = DB(gua.conn_ra["host"], gua.conn_ra["username"], gua.conn_ra["password"], gua.conn_ra["db"], gua.conn_ra["port"])
    print "initial length:%s" % len(doraid_views)
    strDate = format_time.timeStamp2DateStr(timeStamp)
    i=0
    with open(config.product_action_outputDir % strDate,"w+") as f:
        for key, views in doraid_views.iteritems() :
            dora_id=key[0]
            parsed_type=key[1]
            uids = list(views["uids"])
            del views["uids"]
            parsed_content=views[parsed_type]
            if len(parsed_content) == 0 :
                continue
            parsed_tags = pt.product_tag(parsed_type, parsed_content)
            if len(parsed_tags.get("country", {})) == 0 and len(parsed_tags.get("city", {})) == 0 and len(parsed_tags.get("poi", {})): 
                continue
            if(type(parsed_content) == set):
                parsed_content=list(parsed_content)
            i += 1
            f.write("%s\t%s\t%s\t%s\t%s\t%s\n" % (strDate,dora_id,json.dumps(uids),parsed_type,json.dumps(parsed_content,ensure_ascii=False),json.dumps(parsed_tags)))
    print "final length:%s" % i
    ra_db.close()

def dump_user_action(theday) :
    doraid_views = {}
    starttime = dateToTimestamp(theday)
#     先删除数据,以免重跑数据时重复录入
    delete_data(theday)
    endtime = starttime + 60*60*24
    print format_time.get_now(), "======begin deal with %s=====" % theday
    print format_time.get_now(), "buy lm"
    doraid_views = buy_lm(starttime, endtime, doraid_views)
    print format_time.get_now(), "book hotel"
    doraid_views = book_hotel(starttime, endtime, doraid_views)
    print format_time.get_now(), "download guide"
    doraid_views = down_guide(starttime, endtime, doraid_views)
    print format_time.get_now(), "ask search"
    doraid_views = ask_search(starttime, endtime, doraid_views)
    print format_time.get_now(), "ask question"
    doraid_views = ask_question(starttime, endtime, doraid_views)
    print format_time.get_now(), "answer question"
    doraid_views = answer_question(starttime, endtime, doraid_views)
    print format_time.get_now(), "place comment"
    doraid_views = place_comment(starttime, endtime, doraid_views)
    print format_time.get_now(), "coplay thread"
    doraid_views = coplay_thread(starttime, endtime, doraid_views)
    print format_time.get_now(), "post travel thread"
    doraid_views = travel_thread(starttime, endtime, doraid_views)
    print format_time.get_now(), "tag doraid action"
    tag_doraid_action(starttime, doraid_views)
    print format_time.get_now(), "======end deal with %s=====" % theday
    
def delete_data(theday) :
    ra_db = DB(gua.conn_ra["host"], gua.conn_ra["username"], gua.conn_ra["password"], gua.conn_ra["db"], gua.conn_ra["port"])
    ra_db.execute("delete from track.placetag_log_tmp where stat_date=%s", (theday.strftime('%Y-%m-%d'),))
    ra_db.close() 

def main(*argv) :
    #默认跑昨天的数据
    if len(argv) == 1 :
        startDay = datetime.datetime.now() - datetime.timedelta(days=1)
        startDay = datetime.datetime(startDay.year, startDay.month, startDay.day)
        dump_user_action(startDay)
    #指定某一天
    elif len(argv) == 2 : 
        startDay = datetime.datetime.strptime(argv[1], "%Y%m%d")
        dump_user_action(startDay)
    #指定起止日期
    elif len(argv) == 3 :
        startDay = datetime.datetime.strptime(argv[1], "%Y%m%d")
        endDay = datetime.datetime.strptime(argv[2], "%Y%m%d")
        while startDay < endDay :
            dump_user_action(startDay)
            startDay += datetime.timedelta(days = 1)

mongo_db = MongoClient(gua.conn_mongo["host"], gua.conn_mongo["db"])

if __name__ == '__main__' :
    main(*sys.argv)
    mongo_db.close()